// HtmlPage.cpp: implementation of the HtmlPage class.
//
//////////////////////////////////////////////////////////////////////

#include "stdafx.h"
#include "HtmlPage.h"

#ifdef _DEBUG
#undef THIS_FILE
static char THIS_FILE[]=__FILE__;
#define new DEBUG_NEW
#endif


bool HtmlPage::GenerateTermsList(char* stream , void *StopMemory, KeyPhraseCollection &objKeyPhraseCollection , bool LearningPhase )
{
	bool allWentWell = true, NotEnglish=true ;	
	int LanguageID = 0, OutCharCodeID = 0, PhraseListSize=0;
	int PhraseIndex = 0, ErrorCode = 0, Result = 0, DetectionCounter = 0;
	double DesiredNumber = 30, Score= 0.0;
	void  *DocumentMemory;
	char *Phrase = NULL, *ErrorMessage = NULL;
	vector< pair<string , int> > TempPhrases;	
	vector< pair<string , int> >::iterator posStrInt;
	FeatureItem objFeatureItem;

/*	ErrorCode = ExtrCreateStopMemory(&StopMemory);
	if (ErrorCode > 0)
	{
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
		return false ;
	}*/
	ErrorCode = ExtrCreateDocumentMemory(&DocumentMemory);
	if (ErrorCode > 0)
	{
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);	       
		return false ;
	}
	ErrorCode = ExtrActivateHTMLFilter(DocumentMemory, StopMemory);
    if (ErrorCode > 0) 
	{
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);	      
		return false ;
    }
	ErrorCode = ExtrSetNumberPhrases(DocumentMemory, DesiredNumber);
	if (ErrorCode > 0) 
	{
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);		    	
		return false ;
	}
    OutCharCodeID = ENC_DOS;
    ErrorCode = ExtrSetOutputCode(DocumentMemory, OutCharCodeID);
    if (ErrorCode > 0)
	{
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);          	
        return false ;
    }	
   	ErrorCode = ExtrReadDocumentBuffer(DocumentMemory, StopMemory,
                        (void *) stream, strlen(stream));
   	if (ErrorCode > 0) 
	{
   		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);   		
   		return false ;
   	}
	ErrorCode = ExtrSignalDocumentEnd(DocumentMemory, StopMemory);
	if (ErrorCode > 0) 
	{
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);	      	
		return false ;
	}
	ErrorCode = ExtrGetDocumentLanguage(DocumentMemory, &LanguageID);
	if (ErrorCode > 0) 
	{
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);	       
		return false ;
	}
	if ( ( LanguageID == 0 ) || ( LanguageID == 1 ) )
	{
		ErrorCode = ExtrGetPhraseListSize(DocumentMemory, &PhraseListSize);
		if (ErrorCode > 0) 
		{
			ExtrGetErrorMessage(ErrorCode, &ErrorMessage);		
			return false ;
		}
		if (PhraseListSize > 0) 
		{
			DetectionCounter = objKeyPhraseCollection.GetSize();
			for ( PhraseIndex = 0; PhraseIndex < PhraseListSize ; PhraseIndex++) 
			{
				ErrorCode = ExtrGetPhraseByIndex(DocumentMemory, PhraseIndex,(void **) &Phrase);
				if (ErrorCode > 0) 
				{
					ExtrGetErrorMessage(ErrorCode, &ErrorMessage);				
					return false ;
				}
				if ( CheckEnglish( Phrase ) )
				{
					pair< string , int > StrIntPair;
					StrIntPair.first.assign( Phrase );
					StrIntPair.second = PhraseIndex;
					TempPhrases.push_back( StrIntPair  );
				}
				else
				{
					NotEnglish =false;
					ExtrClearDocumentMemory( DocumentMemory );
				//	ExtrClearStopMemory(StopMemory);
					return false;
				}
			}
			if ( ( !TempPhrases.empty() ) && ( NotEnglish ) )
			{
				for ( posStrInt = TempPhrases.begin() ; posStrInt != TempPhrases.end() ; posStrInt ++ )
				{
					ErrorCode = ExtrGetScoreByIndex(DocumentMemory, posStrInt->second , &Score);
					if (ErrorCode > 0)
					{
						ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
						ExtrClearDocumentMemory( DocumentMemory );
						//ExtrClearStopMemory(StopMemory);
						return false ;
					}
					if ( LearningPhase == true )
					{
						Result=objKeyPhraseCollection.AddKeyPhraseItem( posStrInt->first );
						objFeatureItem.SetIndex( Result );
					}
					else
					{
						Result=objKeyPhraseCollection.FindKeyPhraseItem( posStrInt->first );
						if ( Result == 0 )
							objFeatureItem.SetIndex( ++DetectionCounter );
						else
							objFeatureItem.SetIndex( Result );
					}
					objFeatureItem.SetWeight( Score );
					objFeatureItem.SetFrequency( FindFrequency( posStrInt->first , stream ) );
					MyFeatures.push_back( objFeatureItem );
				}
			}
		}
		else
		{
			ExtrClearDocumentMemory( DocumentMemory );
			//ExtrClearStopMemory(StopMemory);
			return false;
		}
	}
	else
	{	
		ExtrClearDocumentMemory( DocumentMemory );
		//ExtrClearStopMemory(StopMemory);
		return false;
	}
	ErrorCode = ExtrClearDocumentMemory( DocumentMemory );
	if (ErrorCode > 0)
	{
		ExtrGetErrorMessage( ErrorCode, &ErrorMessage );	       
		return false ;
	}
	//ErrorCode = ExtrClearStopMemory(StopMemory);
	if (ErrorCode > 0)
	{
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);	       
		return false ;
	}
	return allWentWell ;
}


int HtmlPage::FindFrequency( string KeyPhrase , char *stream )
{
	CString CStream( stream  );
	CString CKey( KeyPhrase.c_str()  );
	bool Until = true;
	CStream.MakeLower();
	CKey.MakeLower();
	int position=-1 ,Freq=0 ;
	while( Until )
	{
		position=CStream.Find( CKey , position + 1 );
		if ( position > -1 )
			Freq++;
		else
			Until = false;
	}
	return Freq;	
}

bool HtmlPage::CheckEnglish(char *Phrase)
{
	int PhraseLength = strlen( Phrase );
	for ( int i = 0 ; i < PhraseLength ; i++ )
	{
		/*if ( !( ( Phrase[i] == 32 ) ||
			( Phrase[i] > 64 && Phrase[i] < 91 ) ||
			( Phrase[i] > 96 && Phrase[i] < 123 ) ) )*/
		if ( !(( Phrase[i] > 31 ) && (  Phrase[i] < 127 )))
			return false;
	}
	return true;
}
